{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 思路\n",
    "1. 更完全的文本清洗 (弃置, 不做预处理 ```F1``` 更高 ?)\n",
    "2. 使用 ```universal sentence encoder``` 将每个 ```text``` 向量化\n",
    "3. 以 ```lightGBM``` 构建分类器\n",
    "4. ```hyperopt``` 优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T06:07:45.057838Z",
     "start_time": "2020-03-08T06:07:45.051195Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "from functools import partial\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score, make_scorer\n",
    "from sklearn.model_selection import cross_val_score, ShuffleSplit, KFold\n",
    "from hyperopt import hp, atpe, fmin, Trials, STATUS_OK\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from spellchecker import SpellChecker\n",
    "import tensorflow_hub as hub\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T03:42:33.896425Z",
     "start_time": "2020-03-08T03:42:33.894005Z"
    }
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data/\"\n",
    "TRAIN_DIR = DATA_DIR + \"train.csv\"\n",
    "TEST_DIR = DATA_DIR + \"test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:55:51.337295Z",
     "start_time": "2020-03-08T04:55:51.280869Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size: 7613\n",
      " test size: 3263\n"
     ]
    }
   ],
   "source": [
    "# 训练集, 空格填充空值\n",
    "X_train = pd.read_csv(TRAIN_DIR, encoding=\"utf-8\").fillna(\" \")[[\"text\"]]\n",
    "y_train = pd.read_csv(TRAIN_DIR, encoding=\"utf-8\")[\"target\"]\n",
    "# 测试集, 空格填充空值\n",
    "X_test = pd.read_csv(TEST_DIR, encoding=\"utf-8\").fillna(\" \")[[\"text\"]]\n",
    "\n",
    "print(f\"train size: {len(X_train)}\\n test size: {len(X_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文本清洗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:57:41.721173Z",
     "start_time": "2020-03-08T05:57:41.551788Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/a/34682849\n",
    "def untokenize(words):\n",
    "    \"\"\"Untokenizing a text undoes the tokenizing operation, restoring\n",
    "    punctuation and spaces to the places that people expect them to be.\n",
    "    Ideally, `untokenize(tokenize(text))` should be identical to `text`,\n",
    "    except for line breaks.\n",
    "    \"\"\"\n",
    "    text = ' '.join(words)\n",
    "    step1 = text.replace(\"`` \", '\"').replace(\" ''\", '\"').replace('. . .', '...')\n",
    "    step2 = step1.replace(\" ( \", \" (\").replace(\" ) \", \") \")\n",
    "    step3 = re.sub(r' ([.,:;?!%]+)([ \\'\"`])', r\"\\1\\2\", step2)\n",
    "    step4 = re.sub(r' ([.,:;?!%]+)$', r\"\\1\", step3)\n",
    "    step5 = step4.replace(\" '\", \"'\").replace(\" n't\", \"n't\").replace(\n",
    "        \"can not\", \"cannot\")\n",
    "    step6 = step5.replace(\" ` \", \" '\")\n",
    "    return step6.strip()\n",
    "\n",
    "\n",
    "# https://stackoverflow.com/a/47091490\n",
    "def decontracted(phrase):\n",
    "    \"\"\"Convert contractions like \"can't\" into \"can not\"\n",
    "    \"\"\"\n",
    "    # specific\n",
    "    phrase = re.sub(r\"won\\'t\", \"will not\", phrase)\n",
    "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "\n",
    "    # general\n",
    "    #phrase = re.sub(r\"n't\", \" not\", phrase) # resulted in \"ca not\" when sentence started with \"can't\"\n",
    "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
    "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
    "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "    return phrase\n",
    "\n",
    "\n",
    "# https://github.com/rishabhverma17/sms_slang_translator/blob/master/slang.txt\n",
    "slang_abbrev_dict = {\n",
    "    'AFAIK': 'As Far As I Know',\n",
    "    'AFK': 'Away From Keyboard',\n",
    "    'ASAP': 'As Soon As Possible',\n",
    "    'ATK': 'At The Keyboard',\n",
    "    'ATM': 'At The Moment',\n",
    "    'A3': 'Anytime, Anywhere, Anyplace',\n",
    "    'BAK': 'Back At Keyboard',\n",
    "    'BBL': 'Be Back Later',\n",
    "    'BBS': 'Be Back Soon',\n",
    "    'BFN': 'Bye For Now',\n",
    "    'B4N': 'Bye For Now',\n",
    "    'BRB': 'Be Right Back',\n",
    "    'BRT': 'Be Right There',\n",
    "    'BTW': 'By The Way',\n",
    "    'B4': 'Before',\n",
    "    'B4N': 'Bye For Now',\n",
    "    'CU': 'See You',\n",
    "    'CUL8R': 'See You Later',\n",
    "    'CYA': 'See You',\n",
    "    'FAQ': 'Frequently Asked Questions',\n",
    "    'FC': 'Fingers Crossed',\n",
    "    'FWIW': 'For What It\\'s Worth',\n",
    "    'FYI': 'For Your Information',\n",
    "    'GAL': 'Get A Life',\n",
    "    'GG': 'Good Game',\n",
    "    'GN': 'Good Night',\n",
    "    'GMTA': 'Great Minds Think Alike',\n",
    "    'GR8': 'Great!',\n",
    "    'G9': 'Genius',\n",
    "    'IC': 'I See',\n",
    "    'ICQ': 'I Seek you',\n",
    "    'ILU': 'I Love You',\n",
    "    'IMHO': 'In My Humble Opinion',\n",
    "    'IMO': 'In My Opinion',\n",
    "    'IOW': 'In Other Words',\n",
    "    'IRL': 'In Real Life',\n",
    "    'KISS': 'Keep It Simple, Stupid',\n",
    "    'LDR': 'Long Distance Relationship',\n",
    "    'LMAO': 'Laugh My Ass Off',\n",
    "    'LOL': 'Laughing Out Loud',\n",
    "    'LTNS': 'Long Time No See',\n",
    "    'L8R': 'Later',\n",
    "    'MTE': 'My Thoughts Exactly',\n",
    "    'M8': 'Mate',\n",
    "    'NRN': 'No Reply Necessary',\n",
    "    'OIC': 'Oh I See',\n",
    "    'OMG': 'Oh My God',\n",
    "    'PITA': 'Pain In The Ass',\n",
    "    'PRT': 'Party',\n",
    "    'PRW': 'Parents Are Watching',\n",
    "    'QPSA?': 'Que Pasa?',\n",
    "    'ROFL': 'Rolling On The Floor Laughing',\n",
    "    'ROFLOL': 'Rolling On The Floor Laughing Out Loud',\n",
    "    'ROTFLMAO': 'Rolling On The Floor Laughing My Ass Off',\n",
    "    'SK8': 'Skate',\n",
    "    'STATS': 'Your sex and age',\n",
    "    'ASL': 'Age, Sex, Location',\n",
    "    'THX': 'Thank You',\n",
    "    'TTFN': 'Ta-Ta For Now!',\n",
    "    'TTYL': 'Talk To You Later',\n",
    "    'U': 'You',\n",
    "    'U2': 'You Too',\n",
    "    'U4E': 'Yours For Ever',\n",
    "    'WB': 'Welcome Back',\n",
    "    'WTF': 'What The Fuck',\n",
    "    'WTG': 'Way To Go!',\n",
    "    'WUF': 'Where Are You From?',\n",
    "    'W8': 'Wait',\n",
    "    '7K': 'Sick:-D Laugher'\n",
    "}\n",
    "\n",
    "\n",
    "def unslang(text):\n",
    "    \"\"\"Converts text like \"OMG\" into \"Oh my God\"\n",
    "    \"\"\"\n",
    "    if text.upper() in slang_abbrev_dict.keys():\n",
    "        return slang_abbrev_dict[text.upper()]\n",
    "    else:\n",
    "        return text\n",
    "\n",
    "\n",
    "# https://gist.github.com/sebleier/554280\n",
    "stopwords = [\n",
    "    \"a\", \"about\", \"above\", \"after\", \"again\", \"against\", \"ain\", \"all\", \"am\",\n",
    "    \"an\", \"and\", \"any\", \"are\", \"aren\", \"aren't\", \"as\", \"at\", \"be\", \"because\",\n",
    "    \"been\", \"before\", \"being\", \"below\", \"between\", \"both\", \"but\", \"by\", \"can\",\n",
    "    \"couldn\", \"couldn't\", \"d\", \"did\", \"didn\", \"didn't\", \"do\", \"does\", \"doesn\",\n",
    "    \"doesn't\", \"doing\", \"don\", \"don't\", \"down\", \"during\", \"each\", \"few\", \"for\",\n",
    "    \"from\", \"further\", \"had\", \"hadn\", \"hadn't\", \"has\", \"hasn\", \"hasn't\", \"have\",\n",
    "    \"haven\", \"haven't\", \"having\", \"he\", \"her\", \"here\", \"hers\", \"herself\", \"him\",\n",
    "    \"himself\", \"his\", \"how\", \"i\", \"if\", \"in\", \"into\", \"is\", \"isn\", \"isn't\",\n",
    "    \"it\", \"it's\", \"its\", \"itself\", \"just\", \"ll\", \"m\", \"ma\", \"me\", \"mightn\",\n",
    "    \"mightn't\", \"more\", \"most\", \"mustn\", \"mustn't\", \"my\", \"myself\", \"needn\",\n",
    "    \"needn't\", \"no\", \"nor\", \"not\", \"now\", \"o\", \"of\", \"off\", \"on\", \"once\",\n",
    "    \"only\", \"or\", \"other\", \"our\", \"ours\", \"ourselves\", \"out\", \"over\", \"own\",\n",
    "    \"re\", \"s\", \"same\", \"shan\", \"shan't\", \"she\", \"she's\", \"should\", \"should've\",\n",
    "    \"shouldn\", \"shouldn't\", \"so\", \"some\", \"such\", \"t\", \"than\", \"that\",\n",
    "    \"that'll\", \"the\", \"their\", \"theirs\", \"them\", \"themselves\", \"then\", \"there\",\n",
    "    \"these\", \"they\", \"this\", \"those\", \"through\", \"to\", \"too\", \"under\", \"until\",\n",
    "    \"up\", \"ve\", \"very\", \"was\", \"wasn\", \"wasn't\", \"we\", \"were\", \"weren\",\n",
    "    \"weren't\", \"what\", \"when\", \"where\", \"which\", \"while\", \"who\", \"whom\", \"why\",\n",
    "    \"will\", \"with\", \"won\", \"won't\", \"wouldn\", \"wouldn't\", \"y\", \"you\", \"you'd\",\n",
    "    \"you'll\", \"you're\", \"you've\", \"your\", \"yours\", \"yourself\", \"yourselves\",\n",
    "    \"could\", \"he'd\", \"he'll\", \"he's\", \"here's\", \"how's\", \"i'd\", \"i'll\", \"i'm\",\n",
    "    \"i've\", \"let's\", \"ought\", \"she'd\", \"she'll\", \"that's\", \"there's\", \"they'd\",\n",
    "    \"they'll\", \"they're\", \"they've\", \"we'd\", \"we'll\", \"we're\", \"we've\",\n",
    "    \"what's\", \"when's\", \"where's\", \"who's\", \"why's\", \"would\", \"able\", \"abst\",\n",
    "    \"accordance\", \"according\", \"accordingly\", \"across\", \"act\", \"actually\",\n",
    "    \"added\", \"adj\", \"affected\", \"affecting\", \"affects\", \"afterwards\", \"ah\",\n",
    "    \"almost\", \"alone\", \"along\", \"already\", \"also\", \"although\", \"always\",\n",
    "    \"among\", \"amongst\", \"announce\", \"another\", \"anybody\", \"anyhow\", \"anymore\",\n",
    "    \"anyone\", \"anything\", \"anyway\", \"anyways\", \"anywhere\", \"apparently\",\n",
    "    \"approximately\", \"arent\", \"arise\", \"around\", \"aside\", \"ask\", \"asking\",\n",
    "    \"auth\", \"available\", \"away\", \"awfully\", \"b\", \"back\", \"became\", \"become\",\n",
    "    \"becomes\", \"becoming\", \"beforehand\", \"begin\", \"beginning\", \"beginnings\",\n",
    "    \"begins\", \"behind\", \"believe\", \"beside\", \"besides\", \"beyond\", \"biol\",\n",
    "    \"brief\", \"briefly\", \"c\", \"ca\", \"came\", \"cannot\", \"can't\", \"cause\", \"causes\",\n",
    "    \"certain\", \"certainly\", \"co\", \"com\", \"come\", \"comes\", \"contain\",\n",
    "    \"containing\", \"contains\", \"couldnt\", \"date\", \"different\", \"done\",\n",
    "    \"downwards\", \"due\", \"e\", \"ed\", \"edu\", \"effect\", \"eg\", \"eight\", \"eighty\",\n",
    "    \"either\", \"else\", \"elsewhere\", \"end\", \"ending\", \"enough\", \"especially\",\n",
    "    \"et\", \"etc\", \"even\", \"ever\", \"every\", \"everybody\", \"everyone\", \"everything\",\n",
    "    \"everywhere\", \"ex\", \"except\", \"f\", \"far\", \"ff\", \"fifth\", \"first\", \"five\",\n",
    "    \"fix\", \"followed\", \"following\", \"follows\", \"former\", \"formerly\", \"forth\",\n",
    "    \"found\", \"four\", \"furthermore\", \"g\", \"gave\", \"get\", \"gets\", \"getting\",\n",
    "    \"give\", \"given\", \"gives\", \"giving\", \"go\", \"goes\", \"gone\", \"got\", \"gotten\",\n",
    "    \"h\", \"happens\", \"hardly\", \"hed\", \"hence\", \"hereafter\", \"hereby\", \"herein\",\n",
    "    \"heres\", \"hereupon\", \"hes\", \"hi\", \"hid\", \"hither\", \"home\", \"howbeit\",\n",
    "    \"however\", \"hundred\", \"id\", \"ie\", \"im\", \"immediate\", \"immediately\",\n",
    "    \"importance\", \"important\", \"inc\", \"indeed\", \"index\", \"information\",\n",
    "    \"instead\", \"invention\", \"inward\", \"itd\", \"it'll\", \"j\", \"k\", \"keep\", \"keeps\",\n",
    "    \"kept\", \"kg\", \"km\", \"know\", \"known\", \"knows\", \"l\", \"largely\", \"last\",\n",
    "    \"lately\", \"later\", \"latter\", \"latterly\", \"least\", \"less\", \"lest\", \"let\",\n",
    "    \"lets\", \"like\", \"liked\", \"likely\", \"line\", \"little\", \"'ll\", \"look\",\n",
    "    \"looking\", \"looks\", \"ltd\", \"made\", \"mainly\", \"make\", \"makes\", \"many\", \"may\",\n",
    "    \"maybe\", \"mean\", \"means\", \"meantime\", \"meanwhile\", \"merely\", \"mg\", \"might\",\n",
    "    \"million\", \"miss\", \"ml\", \"moreover\", \"mostly\", \"mr\", \"mrs\", \"much\", \"mug\",\n",
    "    \"must\", \"n\", \"na\", \"name\", \"namely\", \"nay\", \"nd\", \"near\", \"nearly\",\n",
    "    \"necessarily\", \"necessary\", \"need\", \"needs\", \"neither\", \"never\",\n",
    "    \"nevertheless\", \"new\", \"next\", \"nine\", \"ninety\", \"nobody\", \"non\", \"none\",\n",
    "    \"nonetheless\", \"noone\", \"normally\", \"nos\", \"noted\", \"nothing\", \"nowhere\",\n",
    "    \"obtain\", \"obtained\", \"obviously\", \"often\", \"oh\", \"ok\", \"okay\", \"old\",\n",
    "    \"omitted\", \"one\", \"ones\", \"onto\", \"ord\", \"others\", \"otherwise\", \"outside\",\n",
    "    \"overall\", \"owing\", \"p\", \"page\", \"pages\", \"part\", \"particular\",\n",
    "    \"particularly\", \"past\", \"per\", \"perhaps\", \"placed\", \"please\", \"plus\",\n",
    "    \"poorly\", \"possible\", \"possibly\", \"potentially\", \"pp\", \"predominantly\",\n",
    "    \"present\", \"previously\", \"primarily\", \"probably\", \"promptly\", \"proud\",\n",
    "    \"provides\", \"put\", \"q\", \"que\", \"quickly\", \"quite\", \"qv\", \"r\", \"ran\",\n",
    "    \"rather\", \"rd\", \"readily\", \"really\", \"recent\", \"recently\", \"ref\", \"refs\",\n",
    "    \"regarding\", \"regardless\", \"regards\", \"related\", \"relatively\", \"research\",\n",
    "    \"respectively\", \"resulted\", \"resulting\", \"results\", \"right\", \"run\", \"said\",\n",
    "    \"saw\", \"say\", \"saying\", \"says\", \"sec\", \"section\", \"see\", \"seeing\", \"seem\",\n",
    "    \"seemed\", \"seeming\", \"seems\", \"seen\", \"self\", \"selves\", \"sent\", \"seven\",\n",
    "    \"several\", \"shall\", \"shed\", \"shes\", \"show\", \"showed\", \"shown\", \"showns\",\n",
    "    \"shows\", \"significant\", \"significantly\", \"similar\", \"similarly\", \"since\",\n",
    "    \"six\", \"slightly\", \"somebody\", \"somehow\", \"someone\", \"somethan\",\n",
    "    \"something\", \"sometime\", \"sometimes\", \"somewhat\", \"somewhere\", \"soon\",\n",
    "    \"sorry\", \"specifically\", \"specified\", \"specify\", \"specifying\", \"still\",\n",
    "    \"stop\", \"strongly\", \"sub\", \"substantially\", \"successfully\", \"sufficiently\",\n",
    "    \"suggest\", \"sup\", \"sure\", \"take\", \"taken\", \"taking\", \"tell\", \"tends\", \"th\",\n",
    "    \"thank\", \"thanks\", \"thanx\", \"thats\", \"that've\", \"thence\", \"thereafter\",\n",
    "    \"thereby\", \"thered\", \"therefore\", \"therein\", \"there'll\", \"thereof\",\n",
    "    \"therere\", \"theres\", \"thereto\", \"thereupon\", \"there've\", \"theyd\", \"theyre\",\n",
    "    \"think\", \"thou\", \"though\", \"thoughh\", \"thousand\", \"throug\", \"throughout\",\n",
    "    \"thru\", \"thus\", \"til\", \"tip\", \"together\", \"took\", \"toward\", \"towards\",\n",
    "    \"tried\", \"tries\", \"truly\", \"try\", \"trying\", \"ts\", \"twice\", \"two\", \"u\", \"un\",\n",
    "    \"unfortunately\", \"unless\", \"unlike\", \"unlikely\", \"unto\", \"upon\", \"ups\",\n",
    "    \"us\", \"use\", \"used\", \"useful\", \"usefully\", \"usefulness\", \"uses\", \"using\",\n",
    "    \"usually\", \"v\", \"value\", \"various\", \"'ve\", \"via\", \"viz\", \"vol\", \"vols\",\n",
    "    \"vs\", \"w\", \"want\", \"wants\", \"wasnt\", \"way\", \"wed\", \"welcome\", \"went\",\n",
    "    \"werent\", \"whatever\", \"what'll\", \"whats\", \"whence\", \"whenever\",\n",
    "    \"whereafter\", \"whereas\", \"whereby\", \"wherein\", \"wheres\", \"whereupon\",\n",
    "    \"wherever\", \"whether\", \"whim\", \"whither\", \"whod\", \"whoever\", \"whole\",\n",
    "    \"who'll\", \"whomever\", \"whos\", \"whose\", \"widely\", \"willing\", \"wish\",\n",
    "    \"within\", \"without\", \"wont\", \"words\", \"world\", \"wouldnt\", \"www\", \"x\", \"yes\",\n",
    "    \"yet\", \"youd\", \"youre\", \"z\", \"zero\", \"a's\", \"ain't\", \"allow\", \"allows\",\n",
    "    \"apart\", \"appear\", \"appreciate\", \"appropriate\", \"associated\", \"best\",\n",
    "    \"better\", \"c'mon\", \"c's\", \"cant\", \"changes\", \"clearly\", \"concerning\",\n",
    "    \"consequently\", \"consider\", \"considering\", \"corresponding\", \"course\",\n",
    "    \"currently\", \"definitely\", \"described\", \"despite\", \"entirely\", \"exactly\",\n",
    "    \"example\", \"going\", \"greetings\", \"hello\", \"help\", \"hopefully\", \"ignored\",\n",
    "    \"inasmuch\", \"indicate\", \"indicated\", \"indicates\", \"inner\", \"insofar\",\n",
    "    \"it'd\", \"keep\", \"keeps\", \"novel\", \"presumably\", \"reasonably\", \"second\",\n",
    "    \"secondly\", \"sensible\", \"serious\", \"seriously\", \"sure\", \"t's\", \"third\",\n",
    "    \"thorough\", \"thoroughly\", \"three\", \"well\", \"wonder\", \"a\", \"about\", \"above\",\n",
    "    \"above\", \"across\", \"after\", \"afterwards\", \"again\", \"against\", \"all\",\n",
    "    \"almost\", \"alone\", \"along\", \"already\", \"also\", \"although\", \"always\", \"am\",\n",
    "    \"among\", \"amongst\", \"amoungst\", \"amount\", \"an\", \"and\", \"another\", \"any\",\n",
    "    \"anyhow\", \"anyone\", \"anything\", \"anyway\", \"anywhere\", \"are\", \"around\", \"as\",\n",
    "    \"at\", \"back\", \"be\", \"became\", \"because\", \"become\", \"becomes\", \"becoming\",\n",
    "    \"been\", \"before\", \"beforehand\", \"behind\", \"being\", \"below\", \"beside\",\n",
    "    \"besides\", \"between\", \"beyond\", \"bill\", \"both\", \"bottom\", \"but\", \"by\",\n",
    "    \"call\", \"can\", \"cannot\", \"cant\", \"co\", \"con\", \"could\", \"couldnt\", \"cry\",\n",
    "    \"de\", \"describe\", \"detail\", \"do\", \"done\", \"down\", \"due\", \"during\", \"each\",\n",
    "    \"eg\", \"eight\", \"either\", \"eleven\", \"else\", \"elsewhere\", \"empty\", \"enough\",\n",
    "    \"etc\", \"even\", \"ever\", \"every\", \"everyone\", \"everything\", \"everywhere\",\n",
    "    \"except\", \"few\", \"fifteen\", \"fify\", \"fill\", \"find\", \"first\", \"five\",\n",
    "    \"for\", \"former\", \"formerly\", \"forty\", \"found\", \"four\", \"from\", \"front\",\n",
    "    \"full\", \"further\", \"get\", \"give\", \"go\", \"had\", \"has\", \"hasnt\", \"have\", \"he\",\n",
    "    \"hence\", \"her\", \"here\", \"hereafter\", \"hereby\", \"herein\", \"hereupon\", \"hers\",\n",
    "    \"herself\", \"him\", \"himself\", \"his\", \"how\", \"however\", \"hundred\", \"ie\", \"if\",\n",
    "    \"in\", \"inc\", \"indeed\", \"interest\", \"into\", \"is\", \"it\", \"its\", \"itself\",\n",
    "    \"keep\", \"last\", \"latter\", \"latterly\", \"least\", \"less\", \"ltd\", \"made\",\n",
    "    \"many\", \"may\", \"me\", \"meanwhile\", \"might\", \"mill\", \"mine\", \"more\",\n",
    "    \"moreover\", \"most\", \"mostly\", \"move\", \"much\", \"must\", \"my\", \"myself\",\n",
    "    \"name\", \"namely\", \"neither\", \"never\", \"nevertheless\", \"next\", \"nine\", \"no\",\n",
    "    \"nobody\", \"none\", \"noone\", \"nor\", \"not\", \"nothing\", \"now\", \"nowhere\", \"of\",\n",
    "    \"off\", \"often\", \"on\", \"once\", \"one\", \"only\", \"onto\", \"or\", \"other\",\n",
    "    \"others\", \"otherwise\", \"our\", \"ours\", \"ourselves\", \"out\", \"over\", \"own\",\n",
    "    \"part\", \"per\", \"perhaps\", \"please\", \"put\", \"rather\", \"re\", \"same\", \"see\",\n",
    "    \"seem\", \"seemed\", \"seeming\", \"seems\", \"serious\", \"several\", \"she\", \"should\",\n",
    "    \"show\", \"side\", \"since\", \"sincere\", \"six\", \"sixty\", \"so\", \"some\", \"somehow\",\n",
    "    \"someone\", \"something\", \"sometime\", \"sometimes\", \"somewhere\", \"still\",\n",
    "    \"such\", \"system\", \"take\", \"ten\", \"than\", \"that\", \"the\", \"their\", \"them\",\n",
    "    \"themselves\", \"then\", \"thence\", \"there\", \"thereafter\", \"thereby\",\n",
    "    \"therefore\", \"therein\", \"thereupon\", \"these\", \"they\", \"thickv\", \"thin\",\n",
    "    \"third\", \"this\", \"those\", \"though\", \"three\", \"through\", \"throughout\",\n",
    "    \"thru\", \"thus\", \"to\", \"together\", \"too\", \"top\", \"toward\", \"towards\",\n",
    "    \"twelve\", \"twenty\", \"two\", \"un\", \"under\", \"until\", \"up\", \"upon\", \"us\",\n",
    "    \"very\", \"via\", \"was\", \"we\", \"well\", \"were\", \"what\", \"whatever\", \"when\",\n",
    "    \"whence\", \"whenever\", \"where\", \"whereafter\", \"whereas\", \"whereby\",\n",
    "    \"wherein\", \"whereupon\", \"wherever\", \"whether\", \"which\", \"while\", \"whither\",\n",
    "    \"who\", \"whoever\", \"whole\", \"whom\", \"whose\", \"why\", \"will\", \"with\", \"within\",\n",
    "    \"without\", \"would\", \"yet\", \"you\", \"your\", \"yours\", \"yourself\", \"yourselves\",\n",
    "    \"the\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"m\", \"n\",\n",
    "    \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\", \"w\", \"x\", \"y\", \"z\", \"A\", \"B\", \"C\",\n",
    "    \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\", \"K\", \"L\", \"M\", \"N\", \"O\", \"P\", \"Q\", \"R\",\n",
    "    \"S\", \"T\", \"U\", \"V\", \"W\", \"X\", \"Y\", \"Z\", \"co\", \"op\", \"research-articl\",\n",
    "    \"pagecount\", \"cit\", \"ibid\", \"les\", \"le\", \"au\", \"que\", \"est\", \"pas\", \"vol\",\n",
    "    \"el\", \"los\", \"pp\", \"u201d\", \"well-b\", \"http\", \"volumtype\", \"par\", \"0o\",\n",
    "    \"0s\", \"3a\", \"3b\", \"3d\", \"6b\", \"6o\", \"a1\", \"a2\", \"a3\", \"a4\", \"ab\", \"ac\",\n",
    "    \"ad\", \"ae\", \"af\", \"ag\", \"aj\", \"al\", \"an\", \"ao\", \"ap\", \"ar\", \"av\", \"aw\",\n",
    "    \"ax\", \"ay\", \"az\", \"b1\", \"b2\", \"b3\", \"ba\", \"bc\", \"bd\", \"be\", \"bi\", \"bj\",\n",
    "    \"bk\", \"bl\", \"bn\", \"bp\", \"br\", \"bs\", \"bt\", \"bu\", \"bx\", \"c1\", \"c2\", \"c3\",\n",
    "    \"cc\", \"cd\", \"ce\", \"cf\", \"cg\", \"ch\", \"ci\", \"cj\", \"cl\", \"cm\", \"cn\", \"cp\",\n",
    "    \"cq\", \"cr\", \"cs\", \"ct\", \"cu\", \"cv\", \"cx\", \"cy\", \"cz\", \"d2\", \"da\", \"dc\",\n",
    "    \"dd\", \"de\", \"df\", \"di\", \"dj\", \"dk\", \"dl\", \"do\", \"dp\", \"dr\", \"ds\", \"dt\",\n",
    "    \"du\", \"dx\", \"dy\", \"e2\", \"e3\", \"ea\", \"ec\", \"ed\", \"ee\", \"ef\", \"ei\", \"ej\",\n",
    "    \"el\", \"em\", \"en\", \"eo\", \"ep\", \"eq\", \"er\", \"es\", \"et\", \"eu\", \"ev\", \"ex\",\n",
    "    \"ey\", \"f2\", \"fa\", \"fc\", \"ff\", \"fi\", \"fj\", \"fl\", \"fn\", \"fo\", \"fr\", \"fs\",\n",
    "    \"ft\", \"fu\", \"fy\", \"ga\", \"ge\", \"gi\", \"gj\", \"gl\", \"go\", \"gr\", \"gs\", \"gy\",\n",
    "    \"h2\", \"h3\", \"hh\", \"hi\", \"hj\", \"ho\", \"hr\", \"hs\", \"hu\", \"hy\", \"i\", \"i2\", \"i3\",\n",
    "    \"i4\", \"i6\", \"i7\", \"i8\", \"ia\", \"ib\", \"ic\", \"ie\", \"ig\", \"ih\", \"ii\", \"ij\",\n",
    "    \"il\", \"in\", \"io\", \"ip\", \"iq\", \"ir\", \"iv\", \"ix\", \"iy\", \"iz\", \"jj\", \"jr\",\n",
    "    \"js\", \"jt\", \"ju\", \"ke\", \"kg\", \"kj\", \"km\", \"ko\", \"l2\", \"la\", \"lb\", \"lc\",\n",
    "    \"lf\", \"lj\", \"ln\", \"lo\", \"lr\", \"ls\", \"lt\", \"m2\", \"ml\", \"mn\", \"mo\", \"ms\",\n",
    "    \"mt\", \"mu\", \"n2\", \"nc\", \"nd\", \"ne\", \"ng\", \"ni\", \"nj\", \"nl\", \"nn\", \"nr\",\n",
    "    \"ns\", \"nt\", \"ny\", \"oa\", \"ob\", \"oc\", \"od\", \"of\", \"og\", \"oi\", \"oj\", \"ol\",\n",
    "    \"om\", \"on\", \"oo\", \"oq\", \"or\", \"os\", \"ot\", \"ou\", \"ow\", \"ox\", \"oz\", \"p1\",\n",
    "    \"p2\", \"p3\", \"pc\", \"pd\", \"pe\", \"pf\", \"ph\", \"pi\", \"pj\", \"pk\", \"pl\", \"pm\",\n",
    "    \"pn\", \"po\", \"pq\", \"pr\", \"ps\", \"pt\", \"pu\", \"py\", \"qj\", \"qu\", \"r2\", \"ra\",\n",
    "    \"rc\", \"rd\", \"rf\", \"rh\", \"ri\", \"rj\", \"rl\", \"rm\", \"rn\", \"ro\", \"rq\", \"rr\",\n",
    "    \"rs\", \"rt\", \"ru\", \"rv\", \"ry\", \"s2\", \"sa\", \"sc\", \"sd\", \"se\", \"sf\", \"si\",\n",
    "    \"sj\", \"sl\", \"sm\", \"sn\", \"sp\", \"sq\", \"sr\", \"ss\", \"st\", \"sy\", \"sz\", \"t1\",\n",
    "    \"t2\", \"t3\", \"tb\", \"tc\", \"td\", \"te\", \"tf\", \"th\", \"ti\", \"tj\", \"tl\", \"tm\",\n",
    "    \"tn\", \"tp\", \"tq\", \"tr\", \"ts\", \"tt\", \"tv\", \"tx\", \"ue\", \"ui\", \"uj\", \"uk\",\n",
    "    \"um\", \"un\", \"uo\", \"ur\", \"ut\", \"va\", \"wa\", \"vd\", \"wi\", \"vj\", \"vo\", \"wo\",\n",
    "    \"vq\", \"vt\", \"vu\", \"x1\", \"x2\", \"x3\", \"xf\", \"xi\", \"xj\", \"xk\", \"xl\", \"xn\",\n",
    "    \"xo\", \"xs\", \"xt\", \"xv\", \"xx\", \"y2\", \"yj\", \"yl\", \"yr\", \"ys\", \"yt\", \"zi\", \"zz\"\n",
    "]\n",
    "\n",
    "\n",
    "# Reference : https://gist.github.com/slowkow/7a7f61f495e3dbb7e3d767f97bd7304b\n",
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\n",
    "        \"[\"\n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "        u\"\\U00002702-\\U000027B0\"\n",
    "        u\"\\U000024C2-\\U0001F251\"\n",
    "        \"]+\",\n",
    "        flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)\n",
    "\n",
    "\n",
    "# from: https://www.kaggle.com/shahules/basic-eda-cleaning-and-glove\n",
    "# maybe a bug, it removes question marks?\n",
    "spell = SpellChecker()\n",
    "\n",
    "def correct_spellings(text):\n",
    "    corrected_text = []\n",
    "    misspelled_words = spell.unknown(text.split())\n",
    "    for word in text.split():\n",
    "        if word in misspelled_words:\n",
    "            corrected_text.append(spell.correction(word))\n",
    "        else:\n",
    "            corrected_text.append(word)\n",
    "    return \" \".join(corrected_text)\n",
    "\n",
    "def remove_urls(text):\n",
    "    text = clean(r\"http\\S+\", text)\n",
    "    text = clean(r\"www\\S+\", text)\n",
    "    text = clean(r\"pic.twitter.com\\S+\", text)\n",
    "\n",
    "    return text\n",
    "\n",
    "def clean(reg_exp, text):\n",
    "    text = re.sub(reg_exp, \" \", text)\n",
    "\n",
    "    # replace multiple spaces with one.\n",
    "    text = re.sub('\\s{2,}', ' ', text)\n",
    "\n",
    "    return text\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def clean_all(t, correct_spelling=False, remove_stopwords=False, lemmatize=False):\n",
    "    \n",
    "    # first do bulk cleanup on tokens that don't depend on word tokenization\n",
    "\n",
    "    # remove xml tags\n",
    "    t = clean(r\"<[^>]+>\", t)\n",
    "    t = clean(r\"&lt;\", t)\n",
    "    t = clean(r\"&gt;\", t)\n",
    "\n",
    "    # remove URLs\n",
    "    t = remove_urls(t)\n",
    "\n",
    "    # https://stackoverflow.com/a/35041925\n",
    "    # replace multiple punctuation with single. Ex: !?!?!? would become ?\n",
    "    t = clean(r'[\\?\\.\\!]+(?=[\\?\\.\\!])', t)\n",
    "\n",
    "    t = remove_emoji(t)\n",
    "\n",
    "    # expand common contractions like \"I'm\" \"he'll\"\n",
    "    t = decontracted(t)\n",
    "\n",
    "    # now remove/expand bad patterns per word\n",
    "    words = word_tokenize(t)\n",
    "\n",
    "    # remove stopwords\n",
    "    if remove_stopwords is True:\n",
    "        words = [w for w in words if not w in stopwords]\n",
    "\n",
    "    clean_words = []\n",
    "\n",
    "    for w in words:\n",
    "        # normalize punctuation\n",
    "        w = re.sub(r'&', 'and', w)\n",
    "\n",
    "        # expand slang like OMG = Oh my God\n",
    "        w = unslang(w)\n",
    "\n",
    "        if lemmatize is True:\n",
    "            w = lemmatizer.lemmatize(w)\n",
    "        \n",
    "        clean_words.append(w)\n",
    "\n",
    "    # join the words back into a full string\n",
    "    t = untokenize(clean_words)\n",
    "\n",
    "    if correct_spelling is True:\n",
    "        # this resulted in lots of lost punctuation - omitting for now. Also greatly speeds things up\n",
    "        t = correct_spellings(t)\n",
    "\n",
    "    # finally, remove any non ascii and special characters that made it through\n",
    "    t = clean(r\"[^A-Za-z0-9\\.\\'!\\?,\\$]\", t)\n",
    "    return t\n",
    "\n",
    "\n",
    "def clean_dataframe(df, correct_spelling=False, remove_stopwords=False):\n",
    "    df['clean'] = df.apply(lambda x: clean_all(\n",
    "        x['text'], \n",
    "        correct_spelling=correct_spelling, \n",
    "        remove_stopwords=remove_stopwords), \n",
    "        axis=1\n",
    "    )\n",
    "    return df\n",
    "\n",
    "\n",
    "# https://towardsdatascience.com/make-your-own-super-pandas-using-multiproc-1c04f41944a1\n",
    "def parallelize_dataframe(\n",
    "        df, func, n_cores=2):  # I think Kaggle notebooks only have 2 cores?\n",
    "    df_split = np.array_split(df, n_cores)\n",
    "    pool = Pool(n_cores)\n",
    "    df = pd.concat(pool.map(func, df_split))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:57:44.382589Z",
     "start_time": "2020-03-08T04:57:40.661359Z"
    }
   },
   "outputs": [],
   "source": [
    "# 使用 spell checker 耗时过长\n",
    "# X_train = clean_dataframe(X_train, correct_spelling=True, remove_stopwords=True)\n",
    "\n",
    "X_train = clean_dataframe(X_train, correct_spelling=False, remove_stopwords=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:57:46.090157Z",
     "start_time": "2020-03-08T04:57:46.083528Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>Our Deeds Reason earthquake May ALLAH Forgive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>Forest fire La Ronge Sask. Canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>All residents asked ishelter place' notified o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>13,000 people receive wildfires evacuation ord...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>Just photo Ruby Alaska smoke wildfires pours s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Our Deeds are the Reason of this #earthquake M...   \n",
       "1             Forest fire near La Ronge Sask. Canada   \n",
       "2  All residents asked to 'shelter in place' are ...   \n",
       "3  13,000 people receive #wildfires evacuation or...   \n",
       "4  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "                                               clean  \n",
       "0      Our Deeds Reason earthquake May ALLAH Forgive  \n",
       "1                  Forest fire La Ronge Sask. Canada  \n",
       "2  All residents asked ishelter place' notified o...  \n",
       "3  13,000 people receive wildfires evacuation ord...  \n",
       "4  Just photo Ruby Alaska smoke wildfires pours s...  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# universal sentence encoder 向量化\n",
    "需要提前下载好 ```universal sentence encoder v4``` 模型文件\n",
    "[下载链接](https://hub.tensorflow.google.cn/google/universal-sentence-encoder/4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T03:42:46.847484Z",
     "start_time": "2020-03-08T03:42:46.844655Z"
    }
   },
   "outputs": [],
   "source": [
    "TENSORHUB_DIR = \"/home/liulu/tensorflow_hub/\"\n",
    "UNIVERSAL_SENTENCE_ENCODER = TENSORHUB_DIR + \"universal-sentence-encoder_v4/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T03:42:53.241832Z",
     "start_time": "2020-03-08T03:42:47.521114Z"
    }
   },
   "outputs": [],
   "source": [
    "# encoder 模型\n",
    "emb = hub.load(UNIVERSAL_SENTENCE_ENCODER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:58:23.349648Z",
     "start_time": "2020-03-08T04:58:22.981237Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (7613, 512)\n"
     ]
    }
   ],
   "source": [
    "# 对训练集, 测试集句子编码\n",
    "X_train_embeddings = emb(X_train.clean.values).numpy()\n",
    "# X_test_embeddings = emb(X_test.clean)\n",
    "\n",
    "print(f\"X_train shape: {X_train_embeddings.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:58:26.927756Z",
     "start_time": "2020-03-08T04:58:26.604176Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10683"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "# del stopwords, X_train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lightGBM + hyperopt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:58:29.751069Z",
     "start_time": "2020-03-08T04:58:29.748757Z"
    }
   },
   "outputs": [],
   "source": [
    "# 训练集\n",
    "train_set = lgb.Dataset(X_train_embeddings, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义目标函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T13:33:48.500322Z",
     "start_time": "2020-03-07T13:33:48.496685Z"
    }
   },
   "outputs": [],
   "source": [
    "# def f1_metric(ytrue,preds):\n",
    "#     \"\"\" F1 score, lightGBM 未提供 F1\n",
    "#     \"\"\"\n",
    "# #     return 'f1_score', f1_score((preds>=0.5).astype('int'), ytrue, average='macro'), True\n",
    "#     return f1_score((preds>=0.5).astype('int'), ytrue, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:58:33.506312Z",
     "start_time": "2020-03-08T04:58:33.495949Z"
    }
   },
   "outputs": [],
   "source": [
    "# def objective(params):\n",
    "#     clf = lgb.LGBMClassifier(objective=\"binary\", n_jobs=4, **params)\n",
    "#     # 交叉验证,\n",
    "#     score = cross_val_score(clf, np.array(X_train_embeddings), y_train.values,\n",
    "#                             scoring=make_scorer(f1_metric, greater_is_better=True, needs_proba=False), \n",
    "#                             cv=ShuffleSplit(n_splits=4,test_size=.15), n_jobs=4).mean()\n",
    "#     return score\n",
    "\n",
    "def lgb_f1_score(y_hat, data):\n",
    "    \"\"\" F1 score\n",
    "    \"\"\"\n",
    "    y_true = data.get_label()\n",
    "    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n",
    "    return 'f1', f1_score(y_true, y_hat), True\n",
    "\n",
    "\n",
    "def objective(params, n_folds=5):\n",
    "    \"\"\" 自动调参中的目标函数, 在优化过程中最小化\n",
    "    :param params: 超参数\n",
    "    :n_folds: int, 折数\n",
    "    \"\"\"\n",
    "    params[\"n_estimators\"] = int(params[\"n_estimators\"])\n",
    "    params[\"num_leaves\"] = int(params[\"num_leaves\"])\n",
    "    \n",
    "    # 迭代次数使用params 中的 n_estimators 控制, 将覆盖 cv 中的 num_boost_round 参数\n",
    "    cv_results = lgb.cv(\n",
    "        params=params, train_set=train_set,\n",
    "        nfold=n_folds, \n",
    "        early_stopping_rounds=10, feval=lgb_f1_score,    # lgb 不提供 F1, 此处以 auc 为评价标准\n",
    "        seed=5000, verbose_eval=-1\n",
    "    )\n",
    "    \n",
    "#     best_score = max(cv_results[\"auc-mean\"])\n",
    "    best_score = max(cv_results[\"f1-mean\"])\n",
    "    # 损失应最小化\n",
    "    loss = 1 - best_score\n",
    "    return {\"loss\": loss, \"params\": params, \"status\": STATUS_OK}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义搜索空间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:59:39.293304Z",
     "start_time": "2020-03-08T04:59:39.289897Z"
    }
   },
   "outputs": [],
   "source": [
    "SPACE = {\n",
    "    'num_leaves': hp.quniform('num_leaves', 40, 60, 1),\n",
    "    # 学习率\n",
    "    'learning_rate': hp.loguniform('learning_rate', np.log(0.001), np.log(0.1)),\n",
    "    # 迭代次数\n",
    "    'n_estimators': hp.quniform('n_estimators', 200, 2000, 20),\n",
    "    # 每次迭代随机使用多少比例的特征, 正则化\n",
    "    'feature_fraction': hp.uniform(\"feature_fraction\", 0.5, 0.7),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选择优化算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T04:59:41.709664Z",
     "start_time": "2020-03-08T04:59:41.707330Z"
    }
   },
   "outputs": [],
   "source": [
    "# 自适应 TPE\n",
    "algo = atpe.suggest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:53:45.761914Z",
     "start_time": "2020-03-08T04:59:43.712424Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [54:02<00:00, 16.21s/trial, best loss: 0.2288545649252336]\n"
     ]
    }
   ],
   "source": [
    "best = fmin(\n",
    "    fn=objective,    # 目标函数\n",
    "    space=SPACE,    # 搜索空间\n",
    "    algo=algo,     # 优化算法\n",
    "    max_evals=200,  # 搜索次数\n",
    "    show_progressbar=True, \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 最优参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:55:32.838688Z",
     "start_time": "2020-03-08T05:55:32.835971Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'feature_fraction': 0.6411995672255272,\n",
       " 'learning_rate': 0.06550542142550318,\n",
       " 'n_estimators': 1620.0,\n",
       " 'num_leaves': 50.0}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以最优参数训练分类器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 集成5个模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T15:30:19.170720Z",
     "start_time": "2020-03-07T15:27:15.715509Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7720262618620531"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 交叉验证测试最优参数\n",
    "# np.mean(lgb.cv(\n",
    "#     params=\n",
    "#         {\n",
    "#             'feature_fraction': 0.5016974172479172,\n",
    "#  'learning_rate': 0.03260605135156817,\n",
    "#  'n_estimators': 2000\n",
    "#     }, train_set=train_set, nfold=5,\n",
    "#     feval=lgb_f1_score\n",
    "# )[\"f1-mean\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:56:07.089553Z",
     "start_time": "2020-03-08T05:56:07.087107Z"
    }
   },
   "outputs": [],
   "source": [
    "# lgb 分类器\n",
    "clf = lgb.LGBMClassifier(**\n",
    "{'feature_fraction': 0.6411995672255272,\n",
    " 'learning_rate': 0.06550542142550318,\n",
    " 'n_estimators': 1620,\n",
    " 'num_leaves': 50\n",
    "}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:57:08.588604Z",
     "start_time": "2020-03-08T05:56:08.523811Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               feature_fraction=0.6411995672255272, importance_type='split',\n",
       "               learning_rate=0.06550542142550318, max_depth=-1,\n",
       "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "               n_estimators=1620, n_jobs=-1, num_leaves=50, objective=None,\n",
       "               random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 训练集上训练\n",
    "clf.fit(X_train_embeddings, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在测试集上预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T06:09:24.622910Z",
     "start_time": "2020-03-08T06:09:22.839208Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test = pd.read_csv(TEST_DIR, encoding=\"utf-8\")[[\"text\"]]\n",
    "X_test = clean_dataframe(X_test, False, True)\n",
    "# X_test = emb(X_test[\"clean\"]).numpy()\n",
    "\n",
    "X_test_embeddings = emb(X_test.clean).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:57:49.720007Z",
     "start_time": "2020-03-08T05:57:49.715087Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3263, 512)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:57:51.979808Z",
     "start_time": "2020-03-08T05:57:51.820456Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_test_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T05:57:52.601404Z",
     "start_time": "2020-03-08T05:57:52.597156Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T06:10:19.808194Z",
     "start_time": "2020-03-08T06:10:19.779647Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(TEST_DIR)\n",
    "\n",
    "df = pd.DataFrame({\"id\": test.id, \"target\": result})\n",
    "\n",
    "df.to_csv(\"./submit3.8_5.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 未做文本清洗 测试集 ```F1``` 0.82208\n",
    "- 做了文本清洗 测试集 ```F1``` 0.81595"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "349.091px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "643.722px",
    "left": "1215.43px",
    "right": "20px",
    "top": "94.9716px",
    "width": "460.128px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
